# coding=utf-8

'''
download stock from 163

@author: wd
'''
import os
import time
from htmlUtil import downHTML
import psycopg2
from datetime import datetime, date, timedelta
from wddb import initDb
from stockCommon import delPathFiles
from stockCommon import readStockCodes,readStockIndexCodes

defaultSavePath = os.getcwd()+os.sep+"csv"
defaultStartDate = '20060101'
downloadStockURL = r"http://quotes.money.163.com/service/chddata.html?code=%s%s&start=%s&end=%s&fields=TCLOSE;HIGH;LOW;TOPEN;LCLOSE;CHG;PCHG;TURNOVER;VOTURNOVER;VATURNOVER;TCAP;MCAP"



def queryIndexStartDate(code, con):
    print("queryIndexStartDate code=%s "%code)
    cur = con.cursor()
    sql = "select date from stock_index_data where code='%s'   limit 1 " % code
    cur.execute(sql)
    print("queryIndexStartDate select1 end ")
    rows = cur.fetchall()
    if len(rows) == 0:
        print("code=%s ,row=0 "%code)
        return defaultStartDate
    row = rows[0]
    st = row[0]
    if st == None:
        print("code=%s ,st=None "%code)
        return defaultStartDate
    sql = "select date from stock_index_data where code='%s' order by date desc limit 1 " % code
    cur.execute(sql)
    rows = cur.fetchall()
    if len(rows) == 0:
        return defaultStartDate
    row = rows[0]
    st = row[0]
    # next date
    d = datetime.strptime(st, '%Y-%m-%d')
    d = d+timedelta(days=1)
    st = d.strftime("%Y-%m-%d")
    # check today
    today = datetime.now().strftime("%Y-%m-%d")
    if st > today:
        return ""  # 不需要抓取
    st = st.replace("-", "")
    return st


def queryStockStartDate(code, con):
    cur = con.cursor()
    sql = "select date from stock_data where code='%s' limit 1 " % code
    cur.execute(sql)
    rows = cur.fetchall()
    if len(rows) == 0:
        return defaultStartDate
    sql = "select date from stock_data where code='%s' order by date desc limit 1 " % code
    cur.execute(sql)
    rows = cur.fetchall()
    if len(rows) == 0:
        return defaultStartDate
    row = rows[0]
    st = row[0]
    # next date
    d = datetime.strptime(st, '%Y-%m-%d')
    d = d+timedelta(days=1)
    st = d.strftime("%Y-%m-%d")
    # check today
    today = datetime.now().strftime("%Y-%m-%d")
    if st > today:
        return ""  # 不需要抓取
    st = st.replace("-", "")
    return st


def downAndSaveStock(code, con,savePath):
    print("downAndSaveStock,code=%s"%code)
    st = queryStockStartDate(code, con)
    if st == "":
        print("code=%s,不需要抓取")
        return ""
    # today
    et = time.strftime("%Y%m%d", time.localtime())
    if code.startswith("60"):
        pre = "0"
    elif code.startswith("00"):
        pre = "1"
    elif code.startswith("30"):
        pre = "1"
    else:
        pre = "1"
    u = downloadStockURL % (pre, code, st, et)
    print(u)
    s = downHTML(u, "gbk")
    filename=code.replace("*","_")
    filename = savePath + os.sep + filename + ".csv"
    f=open(filename, 'w')
    f.write(s)
    f.close()
    print("download " + code + " success")
    return filename

def downAndSaveIndex(code, con,savePath):
    st = queryIndexStartDate(code, con)
    if st == "":
        print("code=%s,不需要抓取")
        return ""
    # today
    et = time.strftime("%Y%m%d", time.localtime())
    if code.startswith("00"):
        pre = "0"
    else:
        pre = "1"
    u = downloadStockURL % (pre, code, st, et)
    print(u)
    s = downHTML(u, "gbk")
    filename = savePath + os.sep + code + ".csv"
    f=open(filename, 'w')
    f.write(s)
    f.close()
    print("download index " + code + " success")
    return filename


def snatchStockIndexs(savePath=""):
    if savePath=="":
        savePath=defaultSavePath+os.sep+"index"
    if not os.path.exists(savePath):
        os.makedirs(savePath)
    delPathFiles(savePath)
    con = initDb()
    codes = readStockIndexCodes(con)
    ls = []
    for c in codes:
        print(c)
        c = str(c).strip()
        if c == '':
            continue
        f = downAndSaveIndex(c, con,savePath)
        if f != "":
            ls.append(f)
    con.close()
    return ls

def snatchStocks(savePath=""):
    if savePath=="":
        savePath=defaultSavePath+os.sep+"stock"
    if not os.path.exists(savePath):
        os.makedirs(savePath)
    delPathFiles(savePath)
    con = initDb()
    codes = readStockCodes(con)
    ls = []
    for c in codes:
        print(c)
        c = str(c).strip()
        if c == '':
            continue
        f = downAndSaveStock(c, con,savePath)
        if f != "":
            ls.append(f)
    con.close()
    return ls


if __name__ == "__main__":
    savePath = os.getcwd()+os.sep+"csv"+os.sep+"index"
    snatchStockIndexs(savePath)
    savePath = os.getcwd()+os.sep+"csv"+os.sep+"stock"
    snatchStocks(savePath)
